<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Understanding groups | Elasticsearch Guide [7.7] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch Guide [7.7]">
<link rel="up" href="xpack-rollup.html" title="Rolling up historical data">
<link rel="prev" href="rollup-getting-started.html" title="Getting started with rollups">
<link rel="next" href="rollup-agg-limitations.html" title="Rollup aggregation limitations">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Reference/7.7">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="7.7">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch Guide [7.7]</a></span>
»
<span class="breadcrumb-link"><a href="data-rollup-transform.html">Roll up or transform your data</a></span>
»
<span class="breadcrumb-link"><a href="xpack-rollup.html">Rolling up historical data</a></span>
»
<span class="breadcrumb-node">Understanding groups</span>
</div>
<div class="navheader">
<span class="prev">
<a href="rollup-getting-started.html">« Getting started with rollups</a>
</span>
<span class="next">
<a href="rollup-agg-limitations.html">Rollup aggregation limitations »</a>
</span>
</div>
<div class="section xpack">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="rollup-understanding-groups"></a>Understanding groups<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/understanding-groups.asciidoc">edit</a><a class="xpack_tag" href="/subscriptions"></a>
</h2>
</div></div></div>
<div class="warning admon">
<div class="icon"></div>
<div class="admon_content">
<p>This functionality is experimental and may be changed or removed completely in a future release. Elastic will take a best effort approach to fix any issues, but experimental features are not subject to the support SLA of official GA features.</p>
</div>
</div>
<p>To preserve flexibility, Rollup Jobs are defined based on how future queries may need to use the data.  Traditionally, systems force
the admin to make decisions about what metrics to rollup and on what interval.  E.g. The average of <code class="literal">cpu_time</code> on an hourly basis.  This
is limiting; if, at a future date, the admin wishes to see the average of <code class="literal">cpu_time</code> on an hourly basis <em>and partitioned by `host_name`</em>,
they are out of luck.</p>
<p>Of course, the admin can decide to rollup the <code class="literal">[hour, host]</code> tuple on an hourly basis, but as the number of grouping keys grows, so do the
number of tuples the admin needs to configure.  Furthermore, these <code class="literal">[hours, host]</code> tuples are only useful for hourly rollups…​ daily, weekly,
or monthly rollups all require new configurations.</p>
<p>Rather than force the admin to decide ahead of time which individual tuples should be rolled up, Elasticsearch’s Rollup jobs are configured
based on which groups are potentially useful to future queries.  For example, this configuration:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"groups" : {
  "date_histogram": {
    "field": "timestamp",
    "fixed_interval": "1h",
    "delay": "7d"
  },
  "terms": {
    "fields": ["hostname", "datacenter"]
  },
  "histogram": {
    "fields": ["load", "net_in", "net_out"],
    "interval": 5
  }
}</pre>
</div>
<p>Allows <code class="literal">date_histogram</code>'s to be used on the <code class="literal">"timestamp"</code> field, <code class="literal">terms</code> aggregations to be used on the <code class="literal">"hostname"</code> and <code class="literal">"datacenter"</code>
fields, and <code class="literal">histograms</code> to be used on any of <code class="literal">"load"</code>, <code class="literal">"net_in"</code>, <code class="literal">"net_out"</code> fields.</p>
<p>Importantly, these aggs/fields can be used in any combination.  This aggregation:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"aggs" : {
  "hourly": {
    "date_histogram": {
      "field": "timestamp",
      "fixed_interval": "1h"
    },
    "aggs": {
      "host_names": {
        "terms": {
          "field": "hostname"
        }
      }
    }
  }
}</pre>
</div>
<p>is just as valid as this aggregation:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"aggs" : {
  "hourly": {
    "date_histogram": {
      "field": "timestamp",
      "fixed_interval": "1h"
    },
    "aggs": {
      "data_center": {
        "terms": {
          "field": "datacenter"
        }
      },
      "aggs": {
        "host_names": {
          "terms": {
            "field": "hostname"
          }
        },
        "aggs": {
          "load_values": {
            "histogram": {
              "field": "load",
              "interval": 5
            }
          }
        }
      }
    }
  }
}</pre>
</div>
<p>You’ll notice that the second aggregation is not only substantially larger, it also swapped the position of the terms aggregation on
<code class="literal">"hostname"</code>, illustrating how the order of aggregations does not matter to rollups.  Similarly, while the <code class="literal">date_histogram</code> is required
for rolling up data, it isn’t required while querying (although often used).  For example, this is a valid aggregation for
Rollup Search to execute:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">"aggs" : {
  "host_names": {
    "terms": {
      "field": "hostname"
    }
  }
}</pre>
</div>
<p>Ultimately, when configuring <code class="literal">groups</code> for a job, think in terms of how you might wish to partition data in a query at a future date…​
then include those in the config.  Because Rollup Search allows any order or combination of the grouped fields, you just need to decide
if a field is useful for aggregating later, and how you might wish to use it (terms, histogram, etc).</p>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="rollup-understanding-group-intervals"></a>Calendar vs fixed time intervals<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/understanding-groups.asciidoc">edit</a>
</h3>
</div></div></div>
<p>Each rollup-job must have a date histogram group with a defined interval. Elasticsearch
understands both
<a class="xref" href="search-aggregations-bucket-datehistogram-aggregation.html#calendar_and_fixed_intervals" title="Calendar and fixed intervals">calendar and fixed time intervals</a>. Fixed time
intervals are fairly easy to understand; <code class="literal">60s</code> means sixty seconds. But what
does <code class="literal">1M</code> mean? One month of time depends on which month we are talking about,
some months are longer or shorter than others. This is an example of calendar
time and the duration of that unit depends on context. Calendar units are also
affected by leap-seconds, leap-years, etc.</p>
<p>This is important because the buckets generated by rollup are in either calendar
or fixed intervals and this limits how you can query them later. See
<a class="xref" href="rollup-search-limitations.html#rollup-search-limitations-intervals" title="Requests must be multiples of the config">Requests must be multiples of the config</a>.</p>
<p>We recommend sticking with fixed time intervals, since they are easier to
understand and are more flexible at query time. It will introduce some drift in
your data during leap-events and you will have to think about months in a fixed
quantity (30 days) instead of the actual calendar length. However, it is often
easier than dealing with calendar units at query time.</p>
<p>Multiples of units are always "fixed". For example, <code class="literal">2h</code> is always the fixed
quantity <code class="literal">7200</code> seconds. Single units can be fixed or calendar depending on the
unit:</p>
<div class="informaltable">
<table border="1" cellpadding="4px">
<colgroup>
<col class="col_1">
<col class="col_2">
<col class="col_3">
</colgroup>
<thead>
<tr>
<th align="left" valign="top">Unit</th>
<th align="left" valign="top">Calendar</th>
<th align="left" valign="top">Fixed</th>
</tr>
</thead>
<tbody>
<tr>
<td align="left" valign="top"><p>millisecond</p></td>
<td align="left" valign="top"><p>NA</p></td>
<td align="left" valign="top"><p><code class="literal">1ms</code>, <code class="literal">10ms</code>, etc</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>second</p></td>
<td align="left" valign="top"><p>NA</p></td>
<td align="left" valign="top"><p><code class="literal">1s</code>, <code class="literal">10s</code>, etc</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>minute</p></td>
<td align="left" valign="top"><p><code class="literal">1m</code></p></td>
<td align="left" valign="top"><p><code class="literal">2m</code>, <code class="literal">10m</code>, etc</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>hour</p></td>
<td align="left" valign="top"><p><code class="literal">1h</code></p></td>
<td align="left" valign="top"><p><code class="literal">2h</code>, <code class="literal">10h</code>, etc</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>day</p></td>
<td align="left" valign="top"><p><code class="literal">1d</code></p></td>
<td align="left" valign="top"><p><code class="literal">2d</code>, <code class="literal">10d</code>, etc</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>week</p></td>
<td align="left" valign="top"><p><code class="literal">1w</code></p></td>
<td align="left" valign="top"><p>NA</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>month</p></td>
<td align="left" valign="top"><p><code class="literal">1M</code></p></td>
<td align="left" valign="top"><p>NA</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>quarter</p></td>
<td align="left" valign="top"><p><code class="literal">1q</code></p></td>
<td align="left" valign="top"><p>NA</p></td>
</tr>
<tr>
<td align="left" valign="top"><p>year</p></td>
<td align="left" valign="top"><p><code class="literal">1y</code></p></td>
<td align="left" valign="top"><p>NA</p></td>
</tr>
</tbody>
</table>
</div>
<p>For some units where there are both fixed and calendar, you may need to express
the quantity in terms of the next smaller unit. For example, if you want a fixed
day (not a calendar day), you should specify <code class="literal">24h</code> instead of <code class="literal">1d</code>. Similarly,
if you want fixed hours, specify <code class="literal">60m</code> instead of <code class="literal">1h</code>. This is because the
single quantity entails calendar time, and limits you to querying by calendar
time in the future.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_grouping_limitations_with_heterogeneous_indices"></a>Grouping limitations with heterogeneous indices<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/understanding-groups.asciidoc">edit</a>
</h3>
</div></div></div>
<p>There was previously a limitation in how Rollup could handle indices that had heterogeneous mappings (multiple, unrelated/non-overlapping
mappings).  The recommendation at the time was to configure a separate job per data "type".  For example, you might configure a separate
job for each Beats module that you had enabled (one for <code class="literal">process</code>, another for <code class="literal">filesystem</code>, etc).</p>
<p>This recommendation was driven by internal implementation details that caused document counts to be potentially incorrect if a single "merged"
job was used.</p>
<p>This limitation has since been alleviated.  As of 6.4.0, it is now considered best practice to combine all rollup configurations
into a single job.</p>
<p>As an example, if your index has two types of documents:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{
  "timestamp": 1516729294000,
  "temperature": 200,
  "voltage": 5.2,
  "node": "a"
}</pre>
</div>
<p>and</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{
  "timestamp": 1516729294000,
  "price": 123,
  "title": "Foo"
}</pre>
</div>
<p>the best practice is to combine them into a single rollup job which covers both of these document types, like this:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT _rollup/job/combined
{
    "index_pattern": "data-*",
    "rollup_index": "data_rollup",
    "cron": "*/30 * * * * ?",
    "page_size" :1000,
    "groups" : {
      "date_histogram": {
        "field": "timestamp",
        "fixed_interval": "1h",
        "delay": "7d"
      },
      "terms": {
        "fields": ["node", "title"]
      }
    },
    "metrics": [
        {
            "field": "temperature",
            "metrics": ["min", "max", "sum"]
        },
        {
            "field": "price",
            "metrics": ["avg"]
        }
    ]
}</pre>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="_doc_counts_and_overlapping_jobs"></a>Doc counts and overlapping jobs<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/understanding-groups.asciidoc">edit</a>
</h3>
</div></div></div>
<p>There was previously an issue with document counts on "overlapping" job configurations, driven by the same internal implementation detail.
If there were  two Rollup jobs saving to the same index, where one job is a "subset" of another job, it was possible that document counts
could be incorrect for certain aggregation arrangements.</p>
<p>This issue has also since been eliminated in 6.4.0.</p>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="rollup-getting-started.html">« Getting started with rollups</a>
</span>
<span class="next">
<a href="rollup-agg-limitations.html">Rollup aggregation limitations »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
